import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from pyquery import PyQuery as pq
from pymongo import MongoClient

client = MongoClient('localhost')
db = client['quotes']

options = webdriver.ChromeOptions()
options.add_argument('--headless')
options.add_argument('--disable-gpu')
driver = webdriver.Chrome(chrome_options=options)
driver.get("http://quotes.toscrape.com/page/1/")
wait = WebDriverWait(driver, 10)

def get_page():
    wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,'.col-md-8')))
    html = pq(driver.page_source,parser="html")
    datas = html('.col-md-8 > .quote').items()
    for item in datas:
        result =  {
            'quotes':item('span').eq(0).text(),
            'author':item.find('.author').text(),
            'tags':item('.tags').text()
        }
        save_to_mongo(result)
    if html('nav li.next a'):
        next_url =  driver.current_url[:-8] + html('nav li.next a').attr.href
        print(next_url)
        next_page(next_url)
    else:
        print("Completed!")

def next_page(next_url):
    driver.get(next_url)
    element = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,'nav' )))
    get_page()

def save_to_mongo(result):
    if db['quotes'].insert(result):
        print('saved to mongo')

def main():
    get_page()

if __name__ == '__main__':
    main()